package sparkstreaming.lesson02

import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.streaming.{Seconds, StreamingContext}

/**
  * Created by Administrator on 2018/5/11.
  */
object HDFSWordCount {

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local[2]").setAppName("HDFSWordCount")
    val sc = new SparkContext(conf)
    val ssc = new StreamingContext(sc,Seconds(2))
    /**
      * 数据的输入
      */
    val fileDStream: DStream[String] = ssc.textFileStream("hdfs://hadoop1:9000/streaming")
    /**
      * 数据的处理
      */
    val wordCountDStream: DStream[(String, Int)] = fileDStream.flatMap(_.split(","))
      .map((_, 1))
      .reduceByKey(_ + _)

    /**
      * 数据的输出
      */
    wordCountDStream.print()


    ssc.start()
    ssc.awaitTermination()
    ssc.stop()
  }

}
